{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Cross-validation.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IRsCVlEvAD-t"
      },
      "source": [
        "[参考链接 - skleran - 3.1. Cross-validation: evaluating estimator performance ](https://scikit-learn.org/stable/modules/cross_validation.html#time-series-split)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KFVU9XrbC8J8"
      },
      "source": [
        "### 1.train_test_split 简单分割"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y85_wssoANmd"
      },
      "source": [
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn import datasets\n",
        "from sklearn import svm"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B7HQbB86CWVN",
        "outputId": "d814a7e9-4203-4b61-dfb3-6116265a6511"
      },
      "source": [
        "X, y = datasets.load_iris(return_X_y=True)\n",
        "X.shape, y.shape"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((150, 4), (150,))"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D0nHGtC7CnoY",
        "outputId": "9b4c91f2-8706-4a4d-e39f-0fecd8ec8665"
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=0)\n",
        "\n",
        "print('train: ', X_train.shape, y_train.shape)\n",
        "print('test: ', X_test.shape, y_test.shape)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train:  (90, 4) (90,)\n",
            "test:  (60, 4) (60,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dWRuIBZBDcH9",
        "outputId": "3ca6bd12-eef9-4db5-9a4c-fec94e16111e"
      },
      "source": [
        "clf = svm.SVC(kernel='linear', C=1).fit(X_train, y_train)\n",
        "clf.score(X_test, y_test)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9666666666666667"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FW8e5UspFeQK"
      },
      "source": [
        "##  Computing cross-validated metrics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VleYpvhKDpm1",
        "outputId": "b3a74931-3946-4713-e462-270e199f8c77"
      },
      "source": [
        "from sklearn.model_selection import cross_val_score\n",
        "clf = svm.SVC(kernel='linear', C=1, random_state=42) \n",
        "scores = cross_val_score(clf, X, y, cv=5) # 5折\n",
        "scores"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.96666667, 1.        , 0.96666667, 0.96666667, 1.        ])"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dgmbc-VHF-h-",
        "outputId": "4f31e5ac-d3a5-47b0-f11f-416b5bce0bad"
      },
      "source": [
        "print(\"%0.2f accuracy with a standard deviation of %0.2f\" % (scores.mean(), scores.std()))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.98 accuracy with a standard deviation of 0.02\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "na5My6dLEB7x"
      },
      "source": [
        "### 2. KFold\n",
        "\n",
        "<div align=center>\n",
        "<img src=\"https://scikit-learn.org/stable/_images/grid_search_cross_validation.png\" width=\"420\" height=\"300\" />\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tKBPLyzCGJh0",
        "outputId": "4c0ccea1-129c-49a7-b2a7-b296c49c8374"
      },
      "source": [
        "from sklearn.model_selection import KFold\n",
        "X = ['a', 'b', 'c', 'd']\n",
        "kf = KFold(n_splits=2)\n",
        "\n",
        "for train,test in kf.split(X):\n",
        "  print(\"%s - %s\" % (train, test))\n",
        "\n",
        "# kf2 = KFold(n_splits=3)\n",
        "# for train,test in kf2.split(X):\n",
        "#   print(\"%s - %s\" % (train, test))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2 3] - [0 1]\n",
            "[0 1] - [2 3]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JAiHoIDxJt6s",
        "outputId": "8f16fa77-d54b-409e-96da-599dc03c6414"
      },
      "source": [
        "X = np.array([[0., 0.], [1., 1.], [-1., -1.], [2., 2.]])\n",
        "y = np.array([0, 1, 0, 1])\n",
        "X_train, X_test, y_train, y_test = X[train], X[test], y[train], y[test]\n",
        "X_train, X_test, y_train, y_test"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[0., 0.],\n",
              "        [1., 1.]]), array([[-1., -1.],\n",
              "        [ 2.,  2.]]), array([0, 1]), array([0, 1]))"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dv322R89MHNU"
      },
      "source": [
        "#### 2.1 Repeated K-Fold\n",
        "Example of 2-fold K-Fold repeated 2 times"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pFQpju9MKyLe",
        "outputId": "330bad00-1ff1-4262-e46f-46525e3d4d65"
      },
      "source": [
        "import numpy as np\n",
        "from sklearn.model_selection import RepeatedKFold\n",
        "X = np.array([[1, 2], [3, 4], [1, 2], [3, 4]])\n",
        "random_state = 12883823\n",
        "rkf = RepeatedKFold(n_splits=2, n_repeats=2, random_state=random_state)\n",
        "for train, test in rkf.split(X):\n",
        "    print(\"%s - %s\" % (train, test))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2 3] - [0 1]\n",
            "[0 1] - [2 3]\n",
            "[0 2] - [1 3]\n",
            "[1 3] - [0 2]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gFIuhvLuSuND"
      },
      "source": [
        "#### 2.2 Group k-fold"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MkYVM28BSx3G",
        "outputId": "e5c37a8b-3dfc-45fd-8ae7-7c8f231c9ce2"
      },
      "source": [
        "from sklearn.model_selection import GroupKFold\n",
        "\n",
        "X = [0.1, 0.2, 2.2, 2.4, 2.3, 4.55, 5.8, 8.8, 9, 10]\n",
        "y = [\"a\", \"b\", \"b\", \"b\", \"c\", \"c\", \"c\", \"d\", \"d\", \"d\"]\n",
        "groups = [1, 1, 1, 2, 2, 2, 3, 3, 3, 3]\n",
        "\n",
        "gkf = GroupKFold(n_splits=3)  # <=3\n",
        "for train, test in gkf.split(X, y, groups=groups):\n",
        "    print(\"%s - %s\" % (train, test))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0 1 2 3 4 5] - [6 7 8 9]\n",
            "[0 1 2 6 7 8 9] - [3 4 5]\n",
            "[3 4 5 6 7 8 9] - [0 1 2]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nqiEK-XvNELh"
      },
      "source": [
        "### 3. Leave One Out (LOO)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xANoRmYfMY1K",
        "outputId": "a4bde37c-aee6-4498-990f-6ec2afa05473"
      },
      "source": [
        "from sklearn.model_selection import LeaveOneOut\n",
        "\n",
        "X = [5, 2, 6, 4]\n",
        "loo = LeaveOneOut()\n",
        "for train, test in loo.split(X):\n",
        "    print(\"%s - %s\" % (train, test))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1 2 3] - [0]\n",
            "[0 2 3] - [1]\n",
            "[0 1 3] - [2]\n",
            "[0 1 2] - [3]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JF2LW8iCNljg"
      },
      "source": [
        "#### 3.1  Leave P Out (LPO)\n",
        "Example of Leave-2-Out on a dataset with 6 samples:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W42dd5TcNrPL",
        "outputId": "00caaad6-bbf4-4989-a0d3-77a761d76f65"
      },
      "source": [
        "from sklearn.model_selection import LeavePOut\n",
        "\n",
        "X = np.ones(6)\n",
        "lpo = LeavePOut(p=2)\n",
        "for train, test in lpo.split(X):\n",
        "    print(\"%s - %s\" % (train, test))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2 3 4 5] - [0 1]\n",
            "[1 3 4 5] - [0 2]\n",
            "[1 2 4 5] - [0 3]\n",
            "[1 2 3 5] - [0 4]\n",
            "[1 2 3 4] - [0 5]\n",
            "[0 3 4 5] - [1 2]\n",
            "[0 2 4 5] - [1 3]\n",
            "[0 2 3 5] - [1 4]\n",
            "[0 2 3 4] - [1 5]\n",
            "[0 1 4 5] - [2 3]\n",
            "[0 1 3 5] - [2 4]\n",
            "[0 1 3 4] - [2 5]\n",
            "[0 1 2 5] - [3 4]\n",
            "[0 1 2 4] - [3 5]\n",
            "[0 1 2 3] - [4 5]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6DyY4-4ZPAoR"
      },
      "source": [
        "### 4. ShuffleSplit\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QLkh9vwmNyI_",
        "outputId": "1d414596-1f12-4490-9de8-44b9d0938c8b"
      },
      "source": [
        "from sklearn.model_selection import ShuffleSplit\n",
        "X = np.arange(10)\n",
        "ss = ShuffleSplit(n_splits=5, test_size=0.2, random_state=0)\n",
        "for train_index, test_index in ss.split(X):\n",
        "    print(\"%s - %s\" % (train_index, test_index))"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[4 9 1 6 7 3 0 5] - [2 8]\n",
            "[1 2 9 8 0 6 7 4] - [3 5]\n",
            "[8 4 5 1 0 6 9 7] - [2 3]\n",
            "[9 2 7 5 8 0 3 4] - [6 1]\n",
            "[7 4 1 0 6 8 9 3] - [5 2]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WRpYsH8YR5RC"
      },
      "source": [
        "### 5. Stratified k-fold"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5Qiuu3ymPVFL",
        "outputId": "17df9890-3c92-4bf6-f7bc-86cc3eb09b2c"
      },
      "source": [
        "from sklearn.model_selection import StratifiedKFold, KFold\n",
        "\n",
        "X, y = np.ones((50, 1)), np.hstack(([0] * 45, [1] * 5))\n",
        "skf = StratifiedKFold(n_splits=3)\n",
        "\n",
        "for train, test in skf.split(X, y):\n",
        "    print('train -  {}   |   test -  {}'.format(\n",
        "        np.bincount(y[train]), np.bincount(y[test])))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train -  [30  3]   |   test -  [15  2]\n",
            "train -  [30  3]   |   test -  [15  2]\n",
            "train -  [30  4]   |   test -  [15  1]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AEe-ZpMgSM9p",
        "outputId": "f9824dc7-cd0f-4658-adeb-8134c8e18b6c"
      },
      "source": [
        "kf = KFold(n_splits=3)\n",
        "\n",
        "for train, test in kf.split(X, y):\n",
        "    print('train -  {}   |   test -  {}'.format(\n",
        "        np.bincount(y[train]), np.bincount(y[test])))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train -  [28  5]   |   test -  [17]\n",
            "train -  [28  5]   |   test -  [17]\n",
            "train -  [34]   |   test -  [11  5]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HZ-yJt4wSUrI"
      },
      "source": [
        ""
      ],
      "execution_count": 15,
      "outputs": []
    }
  ]
}